package com.it.spider;

import com.gargoylesoftware.htmlunit.*;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.util.NameValuePair;

import java.io.*;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.sql.Savepoint;
import java.util.*;

/**
 * Created by Administrator on 2015/12/25.
 * 563
 *爬取题库
 */
public class htmlutils {

    public static void main(String args[]) {
      for(int i=100;i<=600;i++){
            System.out.println("编号为:" +i);
            request(i);
        }
    }

    public static void request(int page){
        HtmlPage scorePage = null;
        WebClient webclient = null;
        webclient = new WebClient(BrowserVersion.CHROME);
        webclient.setAjaxController(new NicelyResynchronizingAjaxController());
        WebClientOptions options = webclient.getOptions();
        webclient.waitForBackgroundJavaScript(1000);
        options.setJavaScriptEnabled(true);
        options.setRedirectEnabled(true);
        options.setActiveXNative(false);
        options.setAppletEnabled(false);
        options.setCssEnabled(false);
        options.setDoNotTrackEnabled(false);
        options.setGeolocationEnabled(false);
        options.setUseInsecureSSL(true);
        options.setThrowExceptionOnFailingStatusCode(false);
        options.setThrowExceptionOnScriptError(false);

        try {
            HtmlPage resultPage = null;
            //1.登陆
            //http://www.timeep.com:80/login_verify.do
/*
            WebRequest request = new WebRequest(new URL("http://www.timeep.com:80/login_verify.do?userName=nihaoa&userPass=123123"));


           // WebRequest request = new WebRequest(new URL(""));
            request.setHttpMethod(HttpMethod.POST);
            resultPage = webclient.getPage(request);
            System.out.println(resultPage.getUrl());
            System.out.println(resultPage.asText());

                System.out.println(" 登陆成功*************************************************");
*/
            //  System.out.println("未登陆成功");


            //2.登陆过后通过下面地址下载文件
            // http://www.timeep.com/views/ejt/prepackage/prepackageRes_download.do?resourceId=520&resourceFlag=2
            // webclient.getPage("http://www.timeep.com/views/ejt/prepackage/prepackageRes_download.do?resourceId=&resourceFlag=2").getWebResponse();
            // requestF.setHttpMethod(HttpMethod.GET);/*
            //  webclient.
            //System.out.println(p.getWebResponse());
            // System.out.println(" "+p.getUrl().getProtocol()+"              "+p.getUrl().openConnection() );*/
            // webResponse
//            URL url =new URL("http://www.timeep.com/views/ejt/prepackage/prepackageRes_download.do?resourceId=520&resourceFlag=2");
//            HttpURLConnection conn=(HttpURLConnection)url.openConnection();
//            conn.setRequestMethod("GET");
//            conn.getDoInput();
//            int code=conn.getResponseCode();
//            if(code==200){
//               InputStream is= conn.getInputStream();
//                ByteArrayInputStream os=new ByteArrayInputStream();
//            }

            WebRequest requestload = new WebRequest(new URL("http://www.timeep.com/views/ejt/prepackage/prepackageRes_download.do?resourceId="+page+"&resourceFlag=1"));
            Map<String, String> head = new HashMap();
            head.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
            head.put("Accept-Encoding", "gzip, deflate, sdch");
            head.put("Host", "www.timeep.com");
            head.put("Accept-Language", "zh-CN,zh;q=0.8");
            head.put("Connection", "keep-alive");
            head.put("Upgrade-Insecure-Requests","1");
            head.put("User-Agent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36");
//            head.put("Cookie", "vjuids=-2f0c91d8b.151da13b2c6.0.54c1c65b; JSESSIONID=AB2364B5E95D2BC52DFD12F7A2E8BB5B.jvm2; topUserName=nihaoa; topUserPass=; autoLogin=false; CNZZDATA1000279379=1688381193-1451062772-%7C1451265696; vjlast=1451062703.1451266024.13");
            head.put("Cookie","vjuids=-2f0c91d8b.151da13b2c6.0.54c1c65b; JSESSIONID=F4F8F3F86ECE9591CC4BE98C39E574DC.jvm2; topUserName=nihaoa; topUserPass=; autoLogin=false; CNZZDATA1000279379=1688381193-1451062772-%7C1451355454; vjlast=1451062703.1451355386.11");
            requestload.setAdditionalHeaders(head);
            WebResponse response= webclient.loadWebResponse(requestload);
            String status =response.getStatusCode()+"";
            System.out.println("[" + new Date() + "] ......status code: "+status);
            if(webclient.loadWebResponse(requestload).getResponseHeaderValue("Content-Disposition")==null){
                System.out.println("请检查配置***********************************");
                return ;
            }
            String fileName = new String(response.getResponseHeaderValue("Content-Disposition").substring(20).getBytes("ISO8859-1"),"gbk");
            List list=webclient.loadWebResponse(requestload).getResponseHeaders();
            System.out.println("*****************REPONSE HEADER***************************************************************");
            for(int i=0;i<list.size();i++){
                System.out.println("**"+list.get(i));
            }
            System.out.println("**********************************************************************************************");
            //拿到该页面响应的InputStream()
            InputStream is = response.getContentAsStream();
            System.out.println("file name:" + fileName);

            //下载文件
            load(is,fileName);

        } catch (MalformedURLException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    //下载文件功能
    public static void load(InputStream is,String fileName) {

        File savePath = new File("F://GAOKAO3/" );
        File file=null;
        if (!savePath.exists()) {
            savePath.mkdir();
        }
        file = new File(savePath, fileName);
        if (!file.exists()) {
            try {
                file.createNewFile();
                System.out.println("[" + new Date() + "]+   文件路径创建成功！！！！" + file.getPath());
            } catch (IOException e) {
                System.out.println("[" + new Date() + "]   文件路径创建失败！!请检查路径");
            }
        }
        FileOutputStream fos = null;
        try {
            fos = new FileOutputStream(file.getPath());
            byte[] buffer = new byte[3096];
            int temp = 0;
            System.out.println("[" + new Date() + "]    开始下载文件"+fileName);
            while (true) {
                temp = is.read(buffer, 0, buffer.length);
                if (temp == -1) {
                    break;
                }
                fos.write(buffer, 0, temp);
            }
            System.out.println("[" + new Date() + "]  文件下载完毕:" +fileName +"  download suceessed");
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            try {
                is.close();
                fos.close();
            } catch (Exception e2) {
                System.out.println(e2);;
            }
        }
    }
}